#!/usr/bin/env python
import single_process.init

from tzutil.unescape import unescape
from iwin.db import PG
from iwin.spider.req.xueqiu import requests
from os.path import join, dirname
from time import time

URL = "https://xueqiu.com/stock/cata/stocklist.json?page=%s&size=100&order=desc&orderby=percent&type=%s"
MARKET = dict(CN="11,12", HK="30", US="0,1,2,3")

TABLE = "stock.code"


def fetch_market(market):
    page = 1
    while 1:
        url = URL % (page, market)
        # print(url)
        stocks = requests.get(url).json()['stocks']
        if not stocks:
            break
        page += 1
        for i in stocks:
            symbol = i['symbol'].strip()
            name = i['name']
            name = unescape(name).strip()
            yield symbol, name
        # o = Q.xueqiu_market.get(symbol=symbol)

        # if not o:
        #     _id = gid()
        # else:
        #     _id = o._id

        # Q.xueqiu_market.upsert(symbol=symbol)(
        #     name=name, _id=_id, time=int(time()))


def main():
    for country, market_code in MARKET.items():
        # with open(join(dirname(__file__),"stock/%s.txt"%country.lower()),"w") as f:
        for symbol, name in sorted(fetch_market(market_code),
                                   key=lambda x: x[0]):
            market = ""
            if country == "CN" and symbol[2:].isdigit():
                market = symbol[:2]
                symbol = symbol[2:]
                name = name.strip('*')
                if market == "SH":
                    stop = 0
                    for i in "103 090 104 105 106 091".split():
                        if symbol.startswith(i):
                            print(name, symbol)
                            stop = 1
                            break
                    if stop:
                        continue

            elif country == 'HK':
                market = "HK"
                if symbol.startswith('HK'):
                    symbol = symbol[2:]
                symbol = symbol.lstrip('0')

            if PG.select_one(TABLE, dict(code=symbol, market=market)):
                print('EXIST', country, market, symbol)
                continue

            o = PG.insert(
                TABLE,
                dict(name=name, country=country, code=symbol, market=market),
                on_conflict="(market,code) DO UPDATE SET name='%s'" %
                name.replace("'", '"').replace('%', '%%')
            )
            print(market, symbol, name, o)
            # f.write("%s %s\n"%(symbol.strip(), name.strip()))


if __name__ == "__main__":
    main()
